{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d37fec5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Auto-Sklearn cannot be imported.\n"
     ]
    }
   ],
   "source": [
    "from Initialize_and_Train import initialize_and_train\n",
    "from generate_knapsack_data import Gen_Knapsack_data\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dda79988",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = ['DYS', 'BBOpt', 'PertOpt'] #'DYS-Regret', 'SPO+', 'BBOpt', 'PertOpt', 'PertOpt-FY']\n",
    "item_values = np.arange(20,65,5)\n",
    "num_repeats = 5\n",
    "results_test_loss = np.zeros((len(models), len(item_values), num_repeats))\n",
    "results_train_time = np.zeros((len(models), len(item_values), num_repeats))\n",
    "results_time_till_val_loss = np.zeros((len(models), len(item_values), num_repeats))\n",
    "\n",
    "data_type = \"x\"\n",
    "\n",
    "# hard code some data parameters\n",
    "num_knapsack = 2\n",
    "num_feat = 5\n",
    "num_data = 1100\n",
    "\n",
    "# max values\n",
    "max_epochs = 25\n",
    "# max time hardcoded to 30 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fb14fc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating training data for knapsack problem with 2 knapsacks and 20 items\n",
      "Restricted license - for non-production use only - expires 2024-10-28\n",
      "Optimizing for optDataset...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 900/900 [00:00<00:00, 1146.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing for optDataset...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████| 100/100 [00:00<00:00, 797.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing for optDataset...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████| 100/100 [00:00<00:00, 865.63it/s]\n",
      "/Users/danielmckenzie/My-Drive/Research/Fixed_Point_Networks/Diff-Opt-Over-Polytopes-Project/SPO_with_DYS/src/knapsack/dYS_opt_net.py:29: UserWarning: The operator 'aten::linalg_svd' is not currently supported on the MPS backend and will fall back to run on the CPU. This may have performance implications. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/mps/MPSFallback.mm:11.)\n",
      "  U, s, VT = torch.linalg.svd(self.A, full_matrices=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished building dataset\n",
      "\n",
      " Currently training DYS\n",
      "\n",
      "Initial validation loss is  0.4852833151817322\n",
      "epoch:  1 validation loss is  0.10994952917098999 epoch time:  1.43438720703125\n",
      "epoch:  2 validation loss is  0.08687815070152283 epoch time:  1.1937038898468018\n",
      "epoch:  3 validation loss is  0.08795962482690811 epoch time:  1.158055067062378\n",
      "epoch:  4 validation loss is  0.10237923264503479 epoch time:  1.2346560955047607\n",
      "epoch:  5 validation loss is  0.09120403975248337 epoch time:  1.2054672241210938\n",
      "epoch:  6 validation loss is  0.10201874375343323 epoch time:  1.192147970199585\n",
      "epoch:  7 validation loss is  0.10057678073644638 epoch time:  1.135477066040039\n",
      "epoch:  8 validation loss is  0.08795962482690811 epoch time:  1.1953306198120117\n",
      "epoch:  9 validation loss is  0.11247295886278152 epoch time:  1.1923391819000244\n",
      "epoch:  10 validation loss is  0.09444844722747803 epoch time:  1.1543376445770264\n",
      "\n",
      " Currently training BBOpt\n",
      "\n",
      "Num of cores: 1\n",
      "Initial validation loss is  0.4852833151817322\n",
      "\n",
      "Interrupt request received\n",
      "epoch:  1 validation loss is  0.5108147263526917 epoch time:  1.367279052734375\n",
      "epoch:  2 validation loss is  0.4848594069480896 epoch time:  1.3709471225738525\n"
     ]
    }
   ],
   "source": [
    "for item_counter, num_item in enumerate(item_values):\n",
    "    knapsack_data_dict = {'num_knapsack': num_knapsack,\n",
    "                          'num_item': num_item,\n",
    "                          'num_feat': num_feat,\n",
    "                          'num_data': num_data\n",
    "                         } \n",
    "    \n",
    "    # Do the thing\n",
    "    for repeat in range(num_repeats):\n",
    "        # Generate Data\n",
    "        knapsack_dict = Gen_Knapsack_data(num_data, num_feat, num_item, num_knapsack)\n",
    "        for model_counter, model_type in enumerate(models):\n",
    "            train_time, best_test_loss, time_till_best_test_loss = initialize_and_train(knapsack_dict, knapsack_data_dict, model_type, data_type, max_epochs)\n",
    "            results_test_loss[model_counter, item_counter, repeat] = best_test_loss\n",
    "            results_train_time[model_counter, item_counter, repeat] = train_time\n",
    "            results_time_till_val_loss[model_counter, item_counter, repeat] = time_till_best_test_loss\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f72108c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results_test_loss.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a93c416",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Prepare for plotting\n",
    "results_test_loss_mean = np.mean(results_test_loss, axis=2)\n",
    "results_test_loss_max = np.max(results_test_loss, axis=2)\n",
    "results_test_loss_min = np.min(results_test_loss, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78f540fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump to CSV for plotting\n",
    "plotting_dir = \"./plot_data/\"\n",
    "for idx, model_type in enumerate(models):\n",
    "    np.savetxt(plotting_dir + 'results_regret' + model_type+'mean.csv',results_test_loss_mean[idx,:], delimiter=\",\" )\n",
    "    np.savetxt(plotting_dir + 'results_regret' + model_type+'min.csv',results_test_loss_min[idx,:], delimiter=\",\" )\n",
    "    np.savetxt(plotting_dir + 'results_regret' + model_type+'max.csv',results_test_loss_max[idx,:], delimiter=\",\" )\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00be30e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "for idx, model_type in enumerate(models):\n",
    "    plt.plot(item_values, results_test_loss_mean[idx,:], label=model_type)\n",
    "    plt.fill_between(item_values, results_test_loss_min[idx,:], results_test_loss_max[idx,:], alpha=.2)\n",
    "\n",
    "plt.title(\"Best Regret Achieved\")\n",
    "plt.legend()\n",
    "plt.savefig('Knapsack_experiment_varying_num_items_regret_55.png')\n",
    "plt.show()\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "333c9f74",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_time_till_val_loss_mean = np.mean(results_time_till_val_loss, axis=2)\n",
    "results_time_till_val_loss_min = np.min(results_time_till_val_loss, axis=2)\n",
    "results_time_till_val_loss_max = np.max(results_time_till_val_loss, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16d7eba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump to CSV for plotting\n",
    "plotting_dir = \"./plot_data/\"\n",
    "for idx, model_type in enumerate(models):\n",
    "    np.savetxt(plotting_dir + 'results_time_till_best_val_loss' + model_type+'mean.csv',results_time_till_val_loss_mean[idx,:], delimiter=\",\" )\n",
    "    np.savetxt(plotting_dir + 'results_time_till_best_val_loss' + model_type+'min.csv',results_time_till_val_loss_min[idx,:], delimiter=\",\" )\n",
    "    np.savetxt(plotting_dir + 'results_time_till_best_val_loss' + model_type+'max.csv',results_time_till_val_loss_max[idx,:], delimiter=\",\" )\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "145d2f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, model_type in enumerate(models):\n",
    "    print(idx, model_type)\n",
    "    plt.semilogy(item_values, results_time_till_val_loss_mean[idx,:], label=model_type)\n",
    "\n",
    "plt.title(\"Train Time\")\n",
    "plt.legend()\n",
    "plt.savefig('Knapsack_experiment_varying_num_items_train_time_55.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8bed3a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, model_type in enumerate(models):\n",
    "    print(idx, model_type)\n",
    "    plt.plot(item_values, results_time_till_best_loss[idx,:], label=model_type)\n",
    "\n",
    "plt.title(\"Train Time till Best Model\")\n",
    "plt.legend()\n",
    "# plt.savefig('Knapsack_experiment_varying_num_items_train_time.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f136463",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results_time_till_best_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e883fa84",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
